import re
from typing import Any, Dict, List, Text

from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
from rasa.nlu.training_data import Message
from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES


class WhitespaceTokenizer(Tokenizer):

    # 默认配置
    defaults = {
        "intent_tokenization_flag": False,  # 标记以检查是否拆分意图
        "intent_split_symbol": "_",         # 意图应被分割的符号
        "case_sensitive": True,             # 文本将被分词，默认区分大小写
    }

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
        """使用WhitespaceTokenizer框架构造一个新的令牌生成器（tokenizer）。"""

        super().__init__(component_config)    # 初始化父类

        self.case_sensitive = self.component_config["case_sensitive"]

    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
        text = message.get(attribute)

        if not self.case_sensitive:
            text = text.lower()

        # remove 'not a word character' if
        words = re.sub(
            r"[^\w#@&]+(?=\s|$)|"               # 后面有空格或字符串的结尾
            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"     # 字符串前有空格或字符串开头； 不跟数字
            # not in between numbers and not . or @ or & or - or #
            # e.g. 10'000.00 or blabla@gmail.com
            # and not url characters
            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
            " ",
            text,
        ).split()

        # if we removed everything like smiles `:)`, use the whole text as 1 token
        if not words:
            words = [text]

        return self._convert_words_to_tokens(words, text)
